
import json
import time
from openpyxl import load_workbook
import requests
import pandas as pd

keywords = ["便秘"]

yian_list_url = "http://zcy.ckcest.cn/MedicalRecord/search/secondRetval"
yian_detail_url_ref = "http://zcy.ckcest.cn/MedicalRecord/record_detail/record?recordId=%s"
yian_detail_url = "http://zcy.ckcest.cn/MedicalRecord/record_detail/record?recordId=%s&isAuthen=true"
yian_detail_temp_url = ''

cookie = 'Hm_lvt_94917e915f328b2eb14ea08b6fb1549c=1658821706; JSESSIONID=F8CF09C923E5299A1A943FE87583E874; Hm_lpvt_94917e915f328b2eb14ea08b6fb1549c=1658994954; _Collect_ISNEW=1655778642534; _Collect_UD=c59ee50f-8062-4e5e-acec-0dc952d10738; _Collect_CU=f9d57a47603dcd7618f8b4304bda9ef4; _Collect_CU_DEPT=eb163727917cbba1eea208541a643e74; _Collect_SN=0; _YS_userAccect=6cc696b8-a4c0-4663-a69f-6b455b1a3b79; zcyuid=1546; Hm_lvt_8271f1c331cd6e2a04378a28ae17c2f9=1658820416; Hm_lvt_789fd650fa0be6a2a064d019d890b87f=1658820416; Hm_lvt_4c81c993a71c7bc73a0023722c182141=1658821706; acw_sc__v2=62e23677f10067ad647c9ad60f75a4fbbc210df7; ssxmod_itna=QqGx2QitKGqiq0KGHtwnDxc0boGQeDyQ4=OmriCx0yEeGzDAxn40iDto=M0A4Sh+rAaAKC2hq4LbmfPbphCQ7P0afQBr4GIDeKG2DmeDyDi5GRD0IeKbDen=D5xGoDPxDeDAnKDCMrkKDFjwNVQut6QiiDz8NuektMCDDHQDGWFxtDzdTDtMNMKudouw0VfW+4FEDolh6Q8OGKbYExG0hdb7qqt0qPmYyi70D5l0PdlYhd7PDA9c4qzFD===; ssxmod_itna2=QqGx2QitKGqiq0KGHtwnDxc0boGQeDyQ4=OmriDnI3eqDshQQDLG=BOqgQqZtiqidijPSvi5PsadAhC7+onEb+=oAMaKEcQ7QHuvKk3akyHwk8io8qHNZS80i9GPXj2Bo=LtYiAlakNhlu5Da3UDdW3p/3iwD3AvlxbosErStR=sb3bpAedY1nRPPCQuRwKHRBjKduhuBnGeFL2ONafgU=YkZx2MGwjqPSyBfm=jUR=lY1Ops2iAbF7TH9nzE2OcU+vHgm0v5Wo66Oae2WSxKDB2KBaAaQkbvmO01tFjnfTu73jHIauHmev=L0iGvaKklfVhPIB520v/PetaKsnv0OIwxPnnNEELe1I2g2clUcoftO=i3mvDw=tba+Pd+RTK2i3KYZ6733uP=jrqi03NQ5iWqNB4YiDNDwiDoYQ55jfXlmH2mEBh97EXGqipX=Ae5lpiieuQGgYvjapxeYNA4DQK4byu4nxD+5ix=4r4zIke2alDOEhqbGMmkYSMsOGS8u1TiObi=ejs5l+kC=4h5nd=bDxek1vTYiihWwIgCKIqqUj+kwxWk8uQCH6=tDLawd0Q5Bb8y9KRdjX6=R1j1DDFqD+rDxD=; Hm_lpvt_8271f1c331cd6e2a04378a28ae17c2f9=1658994954; Hm_lpvt_789fd650fa0be6a2a064d019d890b87f=1658994954; Hm_lpvt_4c81c993a71c7bc73a0023722c182141=1658994954'

def yian_list(keyword):
    print("获取医案关键词：%s" % keyword)
    data = {
        'keyword': (None,"content:"+keyword)
    }
    headers = {
    # 添加你自己的Cookie
    'Cookie': cookie,
    'Origin': 'http://zcy.ckcest.cn',
    'Pragma': 'no-cache',
    'Host': 'zcy.ckcest.cn',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'
    }
    json =  eval(requests.post(url=yian_list_url,headers=headers,data=data,allow_redirects=False).content.decode('utf-8'))
    list = json['list']
    print(list)
    print("获取关于 %s %s条:" %(keyword, len(list)))
    # 更换为id list
    id_list = []
    for item in list:
        id_list.append(item['recordId'])
    return id_list

def query_record_detail_by_recordId(recordId):
    headers = {
    # 添加你自己的Cookie
    'Cookie': cookie,
    'Origin': 'http://zcy.ckcest.cn',
    'Pragma': 'no-cache',
    'Referer':'http://zcy.ckcest.cn/MedicalRecord/record_detail/record?recordId=%s' % recordId,
    'Host': 'zcy.ckcest.cn',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
    }
    content = requests.get(url="http://zcy.ckcest.cn/MedicalRecord/record_detail/record?recordId=%s&isAuthen=true" % recordId,headers=headers, allow_redirects=False,timeout=3).content.decode('utf-8')
    print("content:"+content)
    # 替换null 为空字符串，避免转换异常 NameError: name 'null' is not defined
    # eval 和 ast 均报异常，使用json.loads
    # content = content.replace('null', ' ')
    # 转成json
    result = json.loads(content)
    print("result:%s" % result)
    return result
    
# # 该接口是以JSESSIONID来进行鉴权的，登录网页获得相应的JSESSIONID即可
# def record_detail_example():
#     # 似乎保留User-Agent用于keepalive即可以
#     headers = {
#         'Cookie': 'Hm_lvt_94917e915f328b2eb14ea08b6fb1549c=1655259781; JSESSIONID=CA3AEB4BFF08C1E32F1AC13FF224D2AF; Hm_lpvt_94917e915f328b2eb14ea08b6fb1549c=1655268610; Hm_lvt_789fd650fa0be6a2a064d019d890b87f=1654682873,1655259160; Hm_lvt_4c81c993a71c7bc73a0023722c182141=1655259781; ssxmod_itna=QqUxnQDQqGqWq0KGHKiQR3ngDRBDg07DOiBGa2Gx0HcoGzDAxn40iDto=k0mY2p0T4KeFg7xKqKbjST4IaWRe4wNLljR55oD84i7DKqibDCqD1D3qDkbWxYAkDt4DTD34DYDi2=DBBLK=D7gU//q6MBf+xxBo/iwQSUuDDCE4DbOpBDB==xBQDXLQ0IWX4xnboW7PPm0vQm0hFKYpNF8Db0Bq57A+efGwx/BvPIcxztg+mkPDAS=O6eD; ssxmod_itna2=QqUxnQDQqGqWq0KGHKiQR3ngDRBDg07DOiBGa2DnIS25Dsao5DLerdvmlSivGNBPvx/0riYxkKg2xYtYYExOOpNnYFg7pIazKMyXj4d3Klu11xnbT7=Twc0d+Ev3ztqXuT3F116nbuGp/O=XWLLRYq3ubjoYK0hHbqrUt3iCSoqoLGYqeWuo2qhpIW04Ypws32LoMgW2bufW71NjjmAQN+03S2DKKYUqqdRmzu9H0d0wLi=qmZAm5DrA3tcpbtLp3zNASEi2CqrrBx0ZtRMS+Cfn0Tj+pOonZb0nRatDVlXo=eiecpjVZwxeeuDhN0h2lqKl+EeIwniqcP5KqobPK1mmeKfEAHYA0j0IA2HC270207w3YEKQvxnetlRP9txxqOhAGjPvAEmYdAQpSDPehqibK5emfhwBBxDpofajlo5KEUZYWhBrX0xlWE7imoRcxaeNjA/hw1iAvbAwrYUY9ZpA9KiX7IxCiox2PRU2FPa8pogNogGyZEY2t/IxF=eD2P5A4DQFK7GF4q5G0WQDThQP22mLCNFYl9RwAhxCeXGxunCA56W/4fMeH/i3N=A1=OO=vWGAo4qbimmZDaPrDfmEjAMxF=7Ix=vp4DFqD+a9Q7gxYD==; zcyuid=1546; Hm_lpvt_4c81c993a71c7bc73a0023722c182141=1655268610; Hm_lpvt_789fd650fa0be6a2a064d019d890b87f=165526861',
#         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36',
#     }
#     res = requests.get(url="http://zcy.ckcest.cn/MedicalRecord/record_detail/record?recordId=63197&isAuthen=true",headers=headers,allow_redirects=False)
#     print("status: %s" % res.status_code)
#     content = res.content.decode('utf-8')
#     print("content:"+content)

def to_csv(list):
    df = pd.DataFrame(list)
    print("----")
    print(df)
    print(df['content'])
    df.to_csv("/Users/feidao/Documents/me/note/development/python/yixin-backend/tools/yian-bianmi.csv")
    try:
        df.to_excel("yian")
        # df.to_csv("yian-3.csv", mode="w", header=None, index=None)
    except Exception as ex:
        print('保存数据异常', ex)

def for_list_record_detail(maxnum):
    for i in range(maxnum):
        query_record_detail_by_recordId(i)
        time.sleep(5)

# 根据关键词查找医案列表，然后根据id查询医案详情，最后录入表单
def spide_with_keyword():
    keywords = ["头晕","眼花"]
    filename = "../database/yian.xlsx"
    for keyword in keywords:
        list = yian_list(keyword)
        print("spider list: %s" % list)
        for recordId in list:
            try:
                item = query_record_detail_by_recordId(recordId)
                url = 'http://zcy.ckcest.cn/MedicalRecord/record_detail?recordId=%s' % recordId
                # item =  {"ret":2,"data":{"recordId":46723,"isExpert":1,"recordTitle":"表里双解法治疗小儿时令、时行病","oldRecordTitle":"null","flag":0,"content":"一儿疹出，腹饱便秘，乃内伤所发也。乃承气汤下之愈。|","doctorId":4062,"doctorName":"魏之琇","tfidf":{},"category":"null","reference":"《续名医类案》","cfIndex":"1 ","ner":"{\"症\": [\"腹饱\", \"便秘\"], \"方\": [\"承气汤\"]}","preList":""}}
                data = item['data']
                print("spider data：", data)
                ws = load_workbook(filename)
                wb = ws.active
                print("write sheet:", wb)
                print([data['recordTitle'],data['content'],data['doctorName'],data['reference'],data['ner']])
                wb.append([data['recordTitle'],data['content'],data['doctorName'],data['reference'],data['ner'],keyword,url])
                ws.save(filename)
            except Exception as ex:
                print(ex)
            else:   
                print("spider success recordId: %s" % recordId)
            finally:
                print("spider recordId end: %s, time sleep 5s" % recordId)
                time.sleep(5)


def append_item_data(filename, data):
    # 使用vs code来创建的xlsx文件会非zip文件错误，需要wps导出一遍
    ws = load_workbook(filename)
    wb = ws.active
    print("write sheet:", wb)
    wb.append([data['recordId'], data['recordTitle'],data['content'],data['doctorName'],data['reference'],data['category'],data['ner'],data['preList']])
    ws.save(filename)

# 按照递增id查找相应的医案
def spide_with_record():
    for i in range(6969, 6970):
        try:
            item =  query_record_detail_by_recordId(i)
            print("spide item data: %s" % item['data'])
            append_item_data("./data/yian_10w.xlsx", item['data'])
        except Exception as ex:
            print(ex)
        else:
            print("")
        finally:
            print("record id: %s" % i)
            time.sleep(1)


if __name__ == '__main__':
    spide_with_record()

